package cn.edu.bjtu.tokenization;

import org.deeplearning4j.text.tokenization.tokenizer.Tokenizer;
import org.deeplearning4j.text.tokenization.tokenizerfactory.DefaultTokenizerFactory;

import java.io.InputStream;

/**
 * Created by alex on 17/6/16.
 */
public class AnsjTokenzierFactory extends DefaultTokenizerFactory {
    boolean onlyNoun = false;
    public AnsjTokenzierFactory(){

    }
    public AnsjTokenzierFactory(boolean onlyNoun){
        this.onlyNoun = onlyNoun;
    }
    @Override
    public Tokenizer create(String toTokenize) {
        Tokenizer t = onlyNoun ? new OnlyNounAnsjTokenizer(toTokenize) :new AnsjTokenizer(toTokenize);
        if(this.getTokenPreProcessor()!=null){
            t.setTokenPreProcessor(getTokenPreProcessor());
        }
        return t;
    }

    @Override
    public Tokenizer create(InputStream toTokenize) {
        Tokenizer t = onlyNoun ? new OnlyNounAnsjTokenizer(toTokenize) :new AnsjTokenizer(toTokenize);
        if(this.getTokenPreProcessor()!=null){
            t.setTokenPreProcessor(getTokenPreProcessor());
        }
        return t;
    }

}
